package com.leaderment.timatt.webmagic.pageprocessor;

import java.net.MalformedURLException;
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Set;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;

public class UpsProcessor implements PageProcessor{
	private String crawlerUrl;
	
	public UpsProcessor(String theUrl){
		this.crawlerUrl = theUrl;
	}
	
	SimpleDateFormat sdf=new SimpleDateFormat("yyyy_MM_dd_HH");
//	sdf.format(crawlDate)
	
	

	int max=7;
    int min=3;
	
	Set<Integer> acceptStatCode=new HashSet<Integer>(){{
	add(200);  
	add(403);
    add(404);
    add(407);
    add(500); 
    add(502);
    add(503);  
    }};
    /**
	 *  部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
	 */
    private Site site = Site.me()
    		//.setDomain("github.com")																			//设置域名，需设置域名后，addCookie才可生效
    		//.addCookie("dotcomt_user","code4craft")															//设置cookies
    		.setDisableCookieManagement(false)																	//禁用cookies
    		.setTimeOut(60000)																					//设置超时时间
    		.setUseGzip(true)																					//设置使用gzip
    		//.setUserAgent(userAgent)																			//设置useragent
    		//.setRetryTimes(3)																					//设置重试次数
    		//.setSleepTime(randomSecond*1000)																	//设置休眠时间
    		//.setCycleRetryTimes(2)
    		.setCharset("UTF-8")
    		.setSleepTime(1000)	
    		.setAcceptStatCode(acceptStatCode)
    		.addHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")	//设置header
    		.addHeader("Accept-Encoding","gzip, deflate")
    		.addHeader("Accept-Language","zh-CN,zh;q=0.8")
    		.addHeader("Upgrade-Insecure-Requests","1")
    		.addHeader("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36");
    
    
    //解析页面的方法
	@SuppressWarnings("unused")
	@Override
    public void process(Page page) {
    	try{
    		if(null == page.getRequest() || null == page.getRawText() || page.getStatusCode()!=200)return;
    		String requestURL=page.getRequest().getUrl();
        	String responseStr=page.getRawText();
        	// 使用JSOUP解析
			Document parse = Jsoup.parse(responseStr);
			String trackingId = "";	String trackingProcess = "";
			String firstVal=""; String secondVal=""; String thirdVal="";
			if (parse!=null) {
				//trackingId Element
				Element trackingIdElt = parse.getElementById("stApp_lblTrackingNumber");
				Elements progressElts =parse.getElementsByClass("ups-progress_current_row");
				if(null != trackingIdElt )trackingId = trackingIdElt.text().split("Number")[1].trim();
				if(progressElts!=null&&progressElts.size()>0){
					if(null != progressElts.select("td") && progressElts.select("td").size()>0) {
						for( int i = 0 ; i<progressElts.select("td").size() ; i++) {
							Element theEl = progressElts.select("td").get(i);
							if(i == 0) {
								firstVal = theEl.text();
							}else if(i == 1) {
								secondVal = theEl.text();
							}else if(i == 2) {
								thirdVal = theEl.text();
							}
						}
					}
				}
				if(!"".equals(firstVal) && !"".equals(secondVal) && !"".equals(thirdVal) )
						page.putField("processStr", secondVal + " " + firstVal + " " + thirdVal);
				if(!"".equals(trackingId))page.putField("trackingId", trackingId);
			}else {
				//存在trackingId则将trackingId放回redis里面重新爬取.
				
			}
    	}catch(Exception e){
    		e.printStackTrace();
    	}
    }
    /**
     * 发现新的URL并放入爬取队列
     * @param page	page实体
     * @param url	失败的URL
     */
    public void addURL(Page page,String url){
		page.addTargetRequest(url);
    }
    
    //从请求URL中获取当前页
    public Integer getCurrentPage(String requestURL){
    	Integer pageNum=1;
    	try {
			URL url=new URL(requestURL);
			String queryString=url.getQuery();
			//System.out.println("queryString:"+queryString);
			String querys[]=queryString.split("&");
			HashMap map=new HashMap();
			for(String query:querys){
				String params[]=query.split("=");
				map.put(params[0], params[1]);
			}
			Object page=map.get("page");
			if(page!=null&&!("").equals(page.toString())){
				//System.out.println("page:"+page);
				pageNum=Integer.parseInt(page.toString());
			}else{
				pageNum=1;
			}
		} catch (MalformedURLException e) {
			e.printStackTrace();
		}
    	return pageNum;
    }
    //替换URL参数
    public static String replaceAccessToken(String url, String name, String accessToken) {
		if (null != url  && null != accessToken && !"".equals(url) && !"".equals(accessToken)) {
			int index = url.indexOf(name + "=");
			if (index != -1) {
				StringBuilder sb = new StringBuilder();
				sb.append(url.substring(0, index)).append(name + "=").append(accessToken);
				int idx = url.indexOf("&", index);
				if (idx != -1) {
					sb.append(url.substring(idx));
				}
				url = sb.toString();
			}
		}
		return url;
	}
    @Override
    public Site getSite() {
        return site;
    }


}
